<!DOCTYPE html>
<html lang="en-us">
<head><head>
    <meta name="google-site-verification" content="9vIieCe-Qpd78QOmBl63rGtIVbhY6sYyuxX3j8XWBA4" />
    <meta name="baidu-site-verification" content="LRrmH41lz7" />
    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="google-site-verification" content="xBT4GhYoi5qRD5tr338pgPM5OWHHIDR6mNg1a3euekI" />
    <meta name="viewport" content="width=device-width, initial-scale=1">
    
    <meta name="baidu-site-verification" content="HGLXRsUXC4" />

    
    <meta name="baidu-site-verification" content="code-ANZvlnN0Xr" />

    
    <meta name="description" content="lan，工程师, 开源爱好者 | 公众号:四颗咖啡豆">
    
    <meta name="keyword"  content="liangyuanpeng|LanLiang|OpenYurt|Knative|Pulsar|Prometheus|Halo||边缘计算kubernetes|Docker|CloudNative|Golang|Rust|Istio|微服务">
    <link rel="shortcut icon" href="img/logo.png">


    
    <meta property="og:image" content="https://res.cloudinary.com/lyp/image/upload/v1544363191/hugo/blog.github.io/743a4e9227e1f14cb24a1eb6db29e183.jpg" />
    <meta name="twitter:image" content="https://res.cloudinary.com/lyp/image/upload/v1544363191/hugo/blog.github.io/743a4e9227e1f14cb24a1eb6db29e183.jpg" />


    <title>prometheus-alertmanager监控告警系统对接钉钉-liangyuanpeng的博客 | liangyuanpeng&#39;s Blog</title>

    <link rel="canonical" href="/post/prometheus-alertmanager-monitoring-dingtalk/">

    <link rel="stylesheet" href="https://res.cloudinary.com/lyp/raw/upload/v1537369740/hugo/css/iDisqus.min.css"/>

    
    <link rel="stylesheet" href="https://res.cloudinary.com/lyp/raw/upload/v1537369744/hugo/css/bootstrap.min.css">

    
    <link rel="stylesheet" href="https://res.cloudinary.com/lyp/raw/upload/v1537369740/hugo/css/hux-blog.min.css">

    
    <link rel="stylesheet" href="https://res.cloudinary.com/lyp/raw/upload/v1537369740/hugo/css/syntax.css">

    
    <link href="//cdnjs.cloudflare.com/ajax/libs/font-awesome/4.6.3/css/font-awesome.min.css" rel="stylesheet" type="text/css">
    
    
    <script src="https://res.cloudinary.com/lyp/raw/upload/v1537369966/hugo/js/jquery.min.js"></script>

    
    <script src="https://res.cloudinary.com/lyp/raw/upload/v1537369966/hugo/js/bootstrap.min.js"></script>

    
    <script src="https://res.cloudinary.com/lyp/raw/upload/v1537369964/hugo/js/hux-blog.min.js"></script>

    
    <script defer src='https://static.cloudflareinsights.com/beacon.min.js' data-cf-beacon='{"token": "f73bc0092aeb491d89984c0eb5a87ac2"}'></script>
</head>

</head>

<nav class="navbar navbar-default navbar-custom navbar-fixed-top">
    <div class="container-fluid">
        
        <div class="navbar-header page-scroll">
            <button type="button" class="navbar-toggle">
                <span class="sr-only">Toggle navigation</span>
                <span class="icon-bar"></span>
                <span class="icon-bar"></span>
                <span class="icon-bar"></span>
            </button>
            <a class="navbar-brand" href="/">Hi,I`m lan</a>
        </div>

        
        
        <div id="huxblog_navbar">
            <div class="navbar-collapse">
                <ul class="nav navbar-nav navbar-right">
                    <li>
                        <a href="/">Home</a>
                    </li>
                    
                    <li>
                        <a href="categories/cloudnative">cloudnative</a>
                    </li>
                    
                    <li>
                        <a href="categories/devops">devops</a>
                    </li>
                    
                    <li>
                        <a href="categories/iot">iot</a>
                    </li>
                    
                    <li>
                        <a href="categories/kubernetes">kubernetes</a>
                    </li>
                    
                    <li>
                        <a href="categories/tech">tech</a>
                    </li>
                    

                    
		            <li>
                        <a href="search">SEARCH <img src="img/search.png" height="15" style="cursor: pointer;"></a>
		            </li>
                    
                </ul>
            </div>
        </div>
        
    </div>
    
</nav>
<script>
    
    
    
    var $body   = document.body;
    var $toggle = document.querySelector('.navbar-toggle');
    var $navbar = document.querySelector('#huxblog_navbar');
    var $collapse = document.querySelector('.navbar-collapse');

    $toggle.addEventListener('click', handleMagic)
    function handleMagic(e){
        if ($navbar.className.indexOf('in') > 0) {
        
            $navbar.className = " ";
            
            setTimeout(function(){
                
                if($navbar.className.indexOf('in') < 0) {
                    $collapse.style.height = "0px"
                }
            },400)
        }else{
        
            $collapse.style.height = "auto"
            $navbar.className += " in";
        }
    }
</script>




<style type="text/css">
    header.intro-header{
        background-image: url('https://res.cloudinary.com/lyp/image/upload/v1544363191/hugo/blog.github.io/743a4e9227e1f14cb24a1eb6db29e183.jpg')
    }
</style>
<header class="intro-header" >
    <div class="container">
        <div class="row">
            <div class="col-lg-8 col-lg-offset-2 col-md-10 col-md-offset-1">
                <div class="post-heading">
                    <div class="tags">
                       
                       <a class="tag" href="/tags/prometheus" title="prometheus">
                           prometheus
                        </a>
                        
                       <a class="tag" href="/tags/docker-compose" title="docker-compose">
                           docker-compose
                        </a>
                        
                       <a class="tag" href="/tags/alertmanager" title="alertmanager">
                           alertmanager
                        </a>
                        
                       <a class="tag" href="/tags/%E9%92%89%E9%92%89" title="钉钉">
                           钉钉
                        </a>
                        
                    </div>
                    <h1>prometheus-alertmanager监控告警系统对接钉钉</h1>
                    <h2 class="subheading"></h2>
                    <span  class="meta">Posted by 梁远鹏 on 2021-03-11
                        
                        <span id="busuanzi_container_page_pv">|<span id="busuanzi_value_page_pv"></span><span>
                            <span id="/post/prometheus-alertmanager-monitoring-dingtalk/" class="leancloud_visitors meta_data_item" data-flag-title="">
    <span class="post-meta-item-icon">
      <span class="octicon octicon-eye"></span> 
    </span>
    <i class="fa fa-eye"></i>
    <span class="old-visitors-count" style="display: none;"></span>
    <span class="leancloud-visitors-count"></span>
</span>






                            阅读 </span></span>|<span class="post-date">共1520字</span>，阅读约<span class="more-meta"> 4 分钟</span>
                        
                    </span>
                </div>
            </div>
        </div>
    </div>
</header>




<article>
    <div class="container">
        <div class="row">

            
            <div class="
                col-lg-8 col-lg-offset-2
                col-md-10 col-md-offset-1
                post-container">

		
                <header>
                <h2>TOC</h2>
                </header>
                <nav id="TableOfContents">
  <ul>
    <li><a href="#下载">下载</a></li>
    <li><a href="#启动webhook程序">启动webhook程序</a></li>
  </ul>

  <ul>
    <li><a href="#告警通知">告警通知</a></li>
    <li><a href="#告警恢复通知">告警恢复通知</a></li>
  </ul>
</nav>
		
		<h1 id="本文还在创作当中将在这几天正式发布">本文还在创作当中，将在这几天正式发布。</h1>
<h1 id="前言">前言</h1>
<p>本文将会讲解alertmanager对接钉钉,并且会演示通过修改metrics值来触发alertmanager进行告警并且展示告警以及告警恢复的情况。</p>
<p>在此之前你可以先查看一下本文的上一篇文章&ndash;&gt;<a href="https://liangyuanpeng.com/post/prometheus-alertmanager-monitoring-quickstart/">prometheus-alertmanager监控告警系统快速开始</a></p>
<p>docker-compose配置以及相关配置文件最终都会贴出来。</p>
<p>AlertManager本身不支持钉钉通知，实现的方式是使用一个实现了钉钉通知API的webhook程序，AlertManager将告警发送到webhook程序，webhook程序再将内容转换成钉钉通知需要的格式发送到钉钉API。</p>
<p>本文阅读前提: 阅读了博文<a href="https://liangyuanpeng.com/post/prometheus-alertmanager-monitoring-quickstart/">prometheus-alertmanager监控告警系统快速开始</a></p>
<h1 id="支持钉钉的webhook程序">支持钉钉的webhook程序</h1>
<p>本文使用的是开源项目<a href="https://github.com/timonwong/prometheus-webhook-dingtalk">https://github.com/timonwong/prometheus-webhook-dingtalk</a>,感谢作者以及贡献者们的辛苦劳动。</p>
<h2 id="下载">下载</h2>
<p>到目前为止,最新版本是<code>1.4.0</code>,你可以下载和本文一致的版本,在钉钉不改变API的情况下我想已经足够使用了,当然你也可以下载最新版本。</p>
<div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-shell" data-lang="shell">wget https://github.com/timonwong/prometheus-webhook-dingtalk/releases/download/v1.4.0/prometheus-webhook-dingtalk-1.4.0.linux-amd64.tar.gz
tar -xvf prometheus-webhook-dingtalk-1.4.0.linux-amd64.tar.gz  
cd prometheus-webhook-dingtalk  
</code></pre></div><p><code>config.yaml</code></p>
<div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-yaml" data-lang="yaml"><span style="color:#f92672">targets</span>:
  <span style="color:#f92672">webhook1</span>:
    <span style="color:#f92672">url</span>: <span style="color:#ae81ff">https://oapi.dingtalk.com/robot/send?access_token={token}</span>
    <span style="color:#75715e"># secret for signature</span>
    <span style="color:#f92672">secret</span>: {<span style="color:#ae81ff">secret}</span>
</code></pre></div><h2 id="启动webhook程序">启动webhook程序</h2>
<div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-shell" data-lang="shell"> ./prometheus-webhook-dingtalk --config.file<span style="color:#f92672">=</span>config.yaml
</code></pre></div><p>到目前为止已经将接收<code>AlertManager</code>告警信息的<code>webhook</code>程序启动好了.</p>
<h1 id="配置alertmanager将告警发送到webhook">配置AlertManager将告警发送到webhook</h1>
<p>如果你已经看了<a href="https://liangyuanpeng.com/post/prometheus-alertmanager-monitoring-quickstart/">prometheus-alertmanager监控告警系统快速开始</a>那么应该知道新增一个webhook告警是多么容易的事情。</p>
<p>首先找到你的AlertManager配置文件,找到<code>receivers</code>的部分,添加以下配置:</p>
<div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-yaml" data-lang="yaml">
<span style="color:#f92672">receivers</span>:
- <span style="color:#f92672">name</span>: <span style="color:#e6db74">&#39;dingtalk&#39;</span>
  <span style="color:#f92672">email_configs</span>: 
    <span style="color:#75715e"># send_resolved:  是否发送恢复告警 </span>
  - <span style="color:#f92672">to</span>: <span style="color:#ae81ff">liangyuanpeng@xxx.com </span> <span style="color:#75715e">#告警发给哪个人接收</span>
</code></pre></div><p>然后找到配置文件<code>route</code>的部分,把<code>receiver</code>的值填写为上述新增的receivers的name,本文是<code>dingtalk</code></p>
<div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-yaml" data-lang="yaml"><span style="color:#f92672">route</span>:
  <span style="color:#ae81ff">...</span>
  <span style="color:#f92672">receiver</span>: <span style="color:#e6db74">&#39;dingtalk&#39;</span>
</code></pre></div><p>重启AlertManager,可以访问AlertManager的status页面来确认配置是否已经生效,例如<code>http://{IP}:{PORT}/#/status</code>.</p>
<h1 id="触发告警">触发告警</h1>
<p>接下来演示一下alertmanager触发告警通知钉钉的情况。使用镜像<code>registry.cn-shenzhen.aliyuncs.com/lan-k8s/lanapp:v0.0.1</code></p>
<p>这是一个SpringBoot程序,更多细节可以在页面<a href="https://liangyuanpeng.com/post/prometheus-alertmanager-monitoring-quickstart/">prometheus-alertmanager监控告警系统快速开始</a>找到XXXX部分查看程序的介绍。</p>
<h2 id="告警通知">告警通知</h2>
<p>设置metrics超过阀值:</p>
<div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-shell" data-lang="shell">curl http://192.168.3.169:8094/index/metrics/30
</code></pre></div><p>注意:将IP替换成实际机器IP,本文后续不再强调.</p>
<p>过了一会可以在钉钉看到机器人发送了告警:<br>
<img src="https://res.cloudinary.com/lyp/image/upload/v1615478134/hugo/blog.github.io/prometheus/dingtalk-alert.png" alt="dingtalk-alert"></p>
<h2 id="告警恢复通知">告警恢复通知</h2>
<p>设置metrics恢复正常值(阀值以下都可以):</p>
<div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-shell" data-lang="shell">curl http://192.168.3.169:8094/index/metrics/5
</code></pre></div><p>过一会钉钉就看到了机器人发送了告警恢复的通知:<br>
<img src="https://res.cloudinary.com/lyp/image/upload/v1615478135/hugo/blog.github.io/prometheus/dingtalk-normal.png" alt="dingtalk-alert-normal"></p>
<h1 id="闭幕">闭幕</h1>
<p>到目前为止一个Metrics在SpringBoot程序中发生变化并且被Prometheus抓取到,并且检测到持续了xx分钟后由AlertManager触发告警到钉钉机器人的这样一个过程就完成了，当然<code>prometheus-webhook-dingtalk</code>这个webhook程序是否还有更高级的玩法可以自己去挖掘挖掘,比如一个告警触发多个钉钉机器人,比如配置多个钉钉机器人,由不同的告警触发不同的钉钉机器人(这在生产时很容易遇到).</p>
<p>并且本文是使用<code>docker-compose</code>部署Prometheus和<code>AlertManager</code>的,在kubernetes部署下是否又有更方便的配置,应该也是我接下来会在博客当中想要写的内容.</p>
<h1 id="注意本文还处于持续创作当中">注意：本文还处于持续创作当中。</h1>


        <h2>微信公众号</h2>
<p>扫描下面的二维码关注我们的微信公众号,第一时间查看最新内容。同时也可以关注我的Github，看看我都在了解什么技术，在页面底部可以找到我的Github。</p>
<img src="https://res.cloudinary.com/lyp/image/upload/v1581729955/hugo/blog.github.io/qrcode_for_sikekafeidou.jpg" alt="wechat-account-qrcode">


                <hr>

                <ul class="pager">
                    
                    <li class="previous">
                        <a href="/post/prometheus-alertmanager-monitoring-quickstart/" data-toggle="tooltip" data-placement="top" title="prometheus-alertmanager监控告警系统快速开始">&larr; prometheus-alertmanager监控告警系统快速开始</a>
                    </li>
                    
                    
                    <li class="next">
                        <a href="/post/metrics-as-a-service/" data-toggle="tooltip" data-placement="top" title="metrics托管服务即将发布">metrics托管服务即将发布 &rarr;</a>
                    </li>
                    
                </ul>

            
<script src="https://utteranc.es/client.js"
repo="liangyuanpeng/liangyuanpeng.github.io"
issue-term="title"
theme="github-light"
crossorigin="anonymous"
async>
</script>



            </div>

            


            
            <div class="
                col-lg-8 col-lg-offset-2
                col-md-10 col-md-offset-1
                sidebar-container">

                
                <section>
                    <hr class="hidden-sm hidden-xs">
                    <h5><a href="/tags/">FEATURED TAGS</a></h5>
                    <div class="tags">
                     
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                            <a href="/tags/blog" title="blog">
                                blog
                            </a>
                        
                    
                        
                    
                        
                            <a href="/tags/chirpstack" title="chirpstack">
                                chirpstack
                            </a>
                        
                    
                        
                    
                        
                            <a href="/tags/ci/cd" title="ci/cd">
                                ci/cd
                            </a>
                        
                    
                        
                    
                        
                            <a href="/tags/cloudnative" title="cloudnative">
                                cloudnative
                            </a>
                        
                    
                        
                            <a href="/tags/cncf" title="cncf">
                                cncf
                            </a>
                        
                    
                        
                    
                        
                            <a href="/tags/docker" title="docker">
                                docker
                            </a>
                        
                    
                        
                            <a href="/tags/docker-compose" title="docker-compose">
                                docker-compose
                            </a>
                        
                    
                        
                    
                        
                    
                        
                    
                        
                            <a href="/tags/fluentd" title="fluentd">
                                fluentd
                            </a>
                        
                    
                        
                    
                        
                    
                        
                            <a href="/tags/halo" title="halo">
                                halo
                            </a>
                        
                    
                        
                            <a href="/tags/hugo" title="hugo">
                                hugo
                            </a>
                        
                    
                        
                            <a href="/tags/image" title="image">
                                image
                            </a>
                        
                    
                        
                    
                        
                            <a href="/tags/iot" title="iot">
                                iot
                            </a>
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                            <a href="/tags/kubernetes" title="kubernetes">
                                kubernetes
                            </a>
                        
                    
                        
                    
                        
                    
                        
                    
                        
                            <a href="/tags/lorawan" title="lorawan">
                                lorawan
                            </a>
                        
                    
                        
                    
                        
                    
                        
                    
                        
                            <a href="/tags/middleware" title="middleware">
                                middleware
                            </a>
                        
                    
                        
                    
                        
                            <a href="/tags/mq" title="mq">
                                mq
                            </a>
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                            <a href="/tags/ops" title="ops">
                                ops
                            </a>
                        
                    
                        
                    
                        
                            <a href="/tags/prometheus" title="prometheus">
                                prometheus
                            </a>
                        
                    
                        
                    
                        
                    
                        
                            <a href="/tags/rpc" title="rpc">
                                rpc
                            </a>
                        
                    
                        
                    
                        
                            <a href="/tags/sofa" title="sofa">
                                sofa
                            </a>
                        
                    
                        
                            <a href="/tags/sofastack" title="sofastack">
                                sofastack
                            </a>
                        
                    
                        
                            <a href="/tags/springboot" title="springboot">
                                springboot
                            </a>
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                    </div>
                </section>



               

                
                
            </div>

            
            
            <div class="
            col-lg- col-lg-offset-2
            col-md-10 col-md-offset-1
            ">
            <section>
                <hr class="hidden-sm hidden-xs">
                

<h2>相关文章</h2>
<ul style="margin-bottom: 25px;">
    
    <li><a href="/post/prometheus-alertmanager-monitoring-quickstart/">prometheus-alertmanager监控告警系统快速开始</a></li>
    
    <li><a href="/post/metrics-as-a-service/">metrics托管服务即将发布</a></li>
    
    <li><a href="/post/deploy-prometheus-operator-kubernetes/">部署Prometheus-Operator监控kubernetes</a></li>
    
</ul>

            </section>
            </div>
            


        </div>
    </div>
</article>






<footer>
    <div class="container">
        <div class="row">
            <div class="col-lg-8 col-lg-offset-2 col-md-10 col-md-offset-1">
                <ul class="list-inline text-center">
                   
                    
                    <li>
                        <a target="_blank" href="https://github.com/liangyuanpeng">
                            <span class="fa-stack fa-lg">
                                <i class="fa fa-circle fa-stack-2x"></i>
                                <i class="fa fa-github fa-stack-1x fa-inverse"></i>
                            </span>
                        </a>
                    </li>
                   
                    
                    <li>
                        <a href="mailto:liangyuanpengem@163.com">
                            <span class="fa-stack fa-lg">
                                <i class="fa fa-circle fa-stack-2x"></i>
                                <i class="fa fa-envelope fa-stack-1x fa-inverse"></i>
                            </span>
                        </a>
                    </li>
		    

                    
                    <li>
                        <a href="https://twitter.com/lan31793328">
                            <span class="fa-stack fa-lg">
                                <i class="fa fa-circle fa-stack-2x"></i>
                                <i class="fa fa-twitter fa-stack-1x fa-inverse"></i>
                            </span>
                        </a>
                    </li>
                    
                    
                    <li>
                        <a target="_blank" href="https://www.zhihu.com/people/liangyuanpeng">
                            <span class="fa-stack fa-lg">
                                <i class="fa fa-circle fa-stack-2x"></i>
                                <i class="fa  fa-stack-1x fa-inverse">知</i>
                            </span>
                        </a>
                    </li>
		    
                    
                    <li>
                        <a target="_blank" href="https://weibo.com/u/1908782280">
                            <span class="fa-stack fa-lg">
                                <i class="fa fa-circle fa-stack-2x"></i>
                                <i class="fa fa-weibo fa-stack-1x fa-inverse"></i>
                            </span>
                        </a>
                    </li>
		    

                    

		    
                    
                    
                </ul>
		<p class="copyright text-muted">
                    Copyright &copy; Hi,I`m lan , 2021
                    <br>
                    <a href="https://themes.gohugo.io/hugo-theme-cleanwhite">CleanWhite Hugo Theme</a> by <a href="https://zhaohuabing.com">Huabing</a> |
                    <iframe
                        style="margin-left: 2px; margin-bottom:-5px;"
                        frameborder="0" scrolling="0" width="100px" height="20px"
                        src="https://ghbtns.com/github-btn.html?user=zhaohuabing&repo=hugo-theme-cleanwhite&type=star&count=true" >
                    </iframe>
                </p>
            </div>
        </div>
    </div>
</footer>




<script>
    function async(u, c) {
      var d = document, t = 'script',
          o = d.createElement(t),
          s = d.getElementsByTagName(t)[0];
      o.src = u;
      if (c) { o.addEventListener('load', function (e) { c(null, e); }, false); }
      s.parentNode.insertBefore(o, s);
    }
</script>






<script>
    
    if($('#tag_cloud').length !== 0){
        async("/js/jquery.tagcloud.js",function(){
            $.fn.tagcloud.defaults = {
                
                color: {start: '#bbbbee', end: '#0085a1'},
            };
            $('#tag_cloud a').tagcloud();
        })
    }
</script>


<script>
    async("https://cdnjs.cloudflare.com/ajax/libs/fastclick/1.0.6/fastclick.js", function(){
        var $nav = document.querySelector("nav");
        if($nav) FastClick.attach($nav);
    })
</script>







<script>
    
    var _baId = 'fad9c137f8ce239f9b323d36c871f8e6';

    
    var _hmt = _hmt || [];
    (function() {
      var hm = document.createElement("script");
      hm.src = "//hm.baidu.com/hm.js?" + _baId;
      var s = document.getElementsByTagName("script")[0];
      s.parentNode.insertBefore(hm, s);
    })();
</script>







</body>
</html>
